#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :deviantart_get_url.py
# @Time      :2023/7/4 
# @Author    :CL
# @email     :1037654919@qq.com


import time
import requests
from bs4 import BeautifulSoup
from utils import MongoDBUtil, mongo_manager
from sqlalchemy import create_engine
import pandas as pd
import os
import json
import shutil
from sqlalchemy import create_engine

engine = create_engine("mysql+pymysql://cl007:ChengLei-0711@127.0.0.1:3306/test?charset=utf8")
from utils import MongoDBUtil, mongo_manager
MongoDBUtil = MongoDBUtil()
images_deviantart = mongo_manager("images_deviantart", db="car_images")
# 代理 pigcha
proxies = {'http': '127.0.0.1:15732',
           'https': '127.0.0.1:15732'}


headers = {
    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0",
    "Accept": "application/json, text/plain, */*",
    "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
    "Accept-Encoding": "gzip, deflate, br",
    "Referer": "https://www.deviantart.com/search/deviations?q=car",
    "Connection": "keep-alive",
    "Sec-Fetch-Dest": "empty",
    "Sec-Fetch-Mode": "cors",
    "Sec-Fetch-Site": "same-origin",
    "TE": "trailers"
}

cookies = {
    "userinfo": "__c5feb6763250224f255c%3B%7B%22username%22%3A%22mashaoming%22%2C%22uniqueid%22%3A%224481e293f9d1d47e024ef899a3bc0f81%22%2C%22dvs9-1%22%3A1%2C%22ab%22%3A%22tao-fp4-1-b-7%7Ctao-pcp-1-b-6%7Ctao-aap-1-b-6%7Ctao-stt-1-b-3%7Ctao-ter-1-a-10%22%7D",
    "_pbjs_userid_consent_data": "3524755945110770",
    "_iiq_fdata": "%7B%22pcid%22%3A%2206778380-4e3d-4c06-a2d9-7bc193f3c431%22%7D",
    "cookie": "e2fcdb84-e53a-4383-82f6-860180ec1863",
    "cto_bundle": "cYQwrl9INUQ2TmRZOGRQTHhEdm1sN3FZaU5hOExOQVUlMkJlcTgyYmwyRDNWcHd2VlBrMzFRUDZNNFElMkJQJTJCMmozejF3bzN2WVYyMEdFQnFPSWRiJTJCVnlMeVdXcndVWU9WdUdNQkI1MU9WS3NUalBnZ2RGWCUyQmFXcjBRWkFtbDZyQ0VqWEdBJTJCc1BpclA0VWY5MnZXcVJWdGM0bm9mUkElM0QlM0Q",
    "cto_bidid": "-0Bfk19aRTF6dTZGWlpHZkoydk5zR0pYTDVtdGo3Q2JVJTJGeW9PbWRtMHJJeUJzZEQ5Vk5oUWolMkZMcWJzeHZ0Yno1Z3VpTlJER24yVnFtcXJGWTdMWXdDSTZ2THJPT2hXcXZqQm91Qlh3eTFtZmwxQ0klMkJFVEw2Zm1vRWx1cnM2NzNTTyUyRmxx",
    "__qca": "P0-483085797-1688019575436",
    "cto_dna_bundle": "YxilK19ObEJNZVNBaVB3Z2dvc2RXTDVjS2kwbmtGbktnRk5pJTJGRWRyVFdvMCUyRjVTZzBnMDN6M2xHakkzaCUyRlNyVmRhZlZ4VFNZZmphUjFvMGt2alRqakxTQmlmZyUzRCUzRA",
    "__gads": "ID=820756d59f47e933:T=1688019594:RT=1688347258:S=ALNI_MbPczHwE5QGYtEd85lIKADluQyWMQ",
    "__gpi": "UID=00000c1c75cdebba:T=1688019594:RT=1688347258:S=ALNI_Ma9DwCCijm2vWYjIGqrbShfHd5r_A",
    "vd": "__d4c8e45a5a309bf9976f%3B%22Bkn%2BVw%2CBkomOX%2CA%2CB%2CA%2C%2CB%2CA%2CB%2CBkomOX%2CBkomOX%2CA%2CA%2CA%2CA%2C13%2CA%2CB%2CA%2CA%2CA%2CA%2CB%2CA%2CA%2C%22",
    "FCNEC": "%5B%5B%22AKsRol_HDx9VBXFNg-7L1KFZrIwjNmKrcP608iMdctTh1nAZom1iVgC4opmjVkk30d1qORL8vccyNyQ-sdo7CZxbSw3A6fgZRa6VrD0d8wG9A_5cA3ir7DGtF1qVe9uQm3ViHFliG_8WCFokP_KnNw3QKr_KqzDHTA%3D%3D%22%5D%2Cnull%2C%5B%5D%5D",
    "_px": "Us/3EWv1lww4nlWsYb/3m7OaC+iUT19csoPTefXCY0qrT7YbpzduHHgy6IDytcU1XbJVGukpvHOCbSHMcJyM4w==:1000:hIGJ9D8MQA7MGVg+c2OW/q5xfIrE13AJqDFpNkQ0q0KFzJHrCFbkGae+wR+PAao/Qx+SIl7bzTx3nsSdnBwsMV39WQlrJUvw/ye51sD3xf24wtC+zOzv1+hiGKY+itmnOnxdGCm83Jc4xndIQOl1Gv5EzNx1QtC4i4qQfsb80w2Ap7jYzNqgYDL148ynmCU9uqpft25PGNVymgy/q0HuueeW8FhepmN1gMgVlzN3rXNz5CxVmXpSmPH66Lr9xcUFaIo2mIzXaUYxCgpQXu7qzw==",
    "_pxvid": "288c607f-193f-11ee-98bd-885e84320cf1",
    "g_state": "{\"i_l\":0}",
    "auth": "__41c9a9096263fe9980c0%3B%22f3b018a25f8c795ee4001f28668f0080%22",
    "auth_secure": "__590f0e80e174697a4589%3B%22cfb8452b13ba59edfad72104bd44dc0e%22",
    "td": "6:1044x532%3B7:960%3B12:1040x777.5%3B20:1919",
    "pxcts": "288c7175-193f-11ee-98bd-4b7a6d776254"
}
cookies[
    'vd'] = '__50bc57dcfe82839ed346%3B%22BkovIi%2CBkovIi%2CA%2CB%2CA%2C%2CB%2CA%2CB%2CBkovIi%2CBkovIi%2CA%2CA%2CA%2CA%2C13%2CA%2CB%2CA%2CA%2CA%2CA%2CB%2CA%2CA%2C%22'


def get_json(kw, cursor=None):
    response = None
    headers = {
        "authority": "www.deviantart.com",
        "accept": "application/json, text/plain, */*",
        "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-US;q=0.7,en-GB;q=0.6",
        "referer": "https://www.deviantart.com/search/deviations?q=car",
        "sec-ch-ua": "\"Microsoft Edge\";v=\"113\", \"Chromium\";v=\"113\", \"Not-A.Brand\";v=\"24\"",
        "sec-ch-ua-mobile": "?1",
        "sec-ch-ua-platform": "\"Android\"",
        "sec-ch-viewport-height": "18604",
        "sec-ch-viewport-width": "980",
        "sec-fetch-dest": "empty",
        "sec-fetch-mode": "cors",
        "sec-fetch-site": "same-origin",
        "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36 Edg/113.0.0.0"
    }
    cookies = {
        "pxcts": "392208e8-196f-11ee-93ad-426c486f6574",
        "_pxvid": "392159bb-196f-11ee-93ad-8c24d60e9489",
        "_px": "zxhz+aHcdMXWMjPary1Z3Xz1qZaSflyGsvaz97VM06rDQjjEDCO4K8XBCiesjSE1hPYDcBAEAWEGhgtiCWdVPg==:1000:K/6ZkoLVvw9ME0ukbqdx+NO+mTQaSeZkhX+vORFweFo3BjOxjH+VBrztc7V2p2F0AiWIBx83jR0rcyPfiYCZUlDDlULalJHNOy1n0o+ZksDG8FzIVtPZ7MkH/s9V0KeFcPizar1RuXJqu/LcNmSre7e2B4oHpIgjPVTEA20e4zOuo3bEaRzQPgyEsJCkj3nKJp7GcYrvMPCe5Zhq3Fljn1injCbbuzDyQ0OY69t7/JmSXFQsaPwv7GBa7LI0KFbiey2uoWxKbPZZo73i9lP3sw==",
        "g_state": "{\"i_l\":0}",
        "auth": "__52a29a3c605a65caae04%3B%22fdc7725533f8165ccb8998d9add38efb%22",
        "auth_secure": "__d02f9013986d367bb161%3B%22d89378bd2dba8a769e97fb92baae06e8%22",
        "userinfo": "__8a0b3d4198f00a79c4a1%3B%7B%22username%22%3A%22mashaoming%22%2C%22uniqueid%22%3A%226b16efb77810db5bcefc393a8fddd03f%22%2C%22dvs9-1%22%3A1%2C%22ab%22%3A%22tao-fp4-1-b-10%7Ctao-pcp-1-b-6%7Ctao-aap-1-b-6%7Ctao-stt-1-b-3%7Ctao-ter-1-a-10%7Ctao-chh-1-a-1%22%7D",
        "td": "6:824x1041%3B7:944%3B8:308%3B12:256x4860%3B20:2272"
    }
    results = []
    mondata = []
    url = "https://www.deviantart.com/_napi/da-browse/api/networkbar/search/deviations"
    params = {
        "q": kw,
        "cursor": cursor,
        "csrf_token": "QDP9IhkX40UaJu29.rx7kww.seDWVJGwGg31sV6f9JW05em3FnoI4ZYlgpNIHnjVPyM"
    }

    while True:
        try:
            response = requests.get(url, headers=headers, proxies=proxies, timeout=10, cookies=cookies,
                                    params=params)
            if response:
                break
        except Exception as e:
            time.sleep(10)
            print('获取数据失败，重试:', e)

    if response.json():
        json_data = response.json()
        # print(json_data)
        try:
            nextCursor = json_data['nextCursor']
        except:
            nextCursor = None
        print(response.status_code)
        print(response.cookies)
        try:
            print(response.cookies.get('vd'))
            cookies['vd'] = response.cookies.get('vd')
        except:
            pass
        deviations = json_data['deviations']
        for data in deviations:
            results.append(data['url'])
            mondata.append({'kw': kw, 'url': data['url'],
                            'deviationId': data['deviationId'], 'title': data['title']})
        # print(results)
        return nextCursor, results, mondata


def save_data_mon(tablename, data):
    datas = {'_id': data[0], 'page': data[0], 'cursor': data[1]}
    try:
        MongoDBUtil.insert_one(tablename, datas)
    except Exception as e:
        print(e)
    pass

def save_data_list(tablename, datas):
    for data in datas:
        data['_id'] = data['deviationId']
        try:
            MongoDBUtil.insert_one(tablename, data)
        except Exception as e:
            # print(e)
            pass


if __name__ == "__main__":
    kws_deviantart = [
        # 'concept car', 'futuristic car','auto', 'automobile', 'car', 'vehicle', 'small car',
        #            'midget car',  'motorcar',
        #               'autobus','minibus','truck','dump truck','cross-country car'
        # 'Automotive', 'super car',
        # 'fantastic car',
        # 'amazing car',
        # 'cool car', 'fabulous car',
        # 'splendid car', 'brilliant car',
        # 'wonderful car', 'fascinating car',
        # 'colorful car',
        # 'beautiful car'
        'future car', 'Hypercar', 'F1','Germancar', 'Ferrari'
                      ]
    for kw in kws_deviantart:
        print('begin:',kw)
        cursor = None
        page = 1
        # cursor="MTQwYWI2MjA9MTg3JjU5MGFjYWQwPTE4NjAmNzgwMDk5ODIlNUIwJTVEPTg5NDQzNzE0NSY3ODAwOTk4MiU1QjElNUQ9ODk0NDM2NDYxJjc4MDA5OTgyJTVCMiU1RD04OTQ0MzcxNjYmNzgwMDk5ODIlNUIzJTVEPTg5NDQzNjc1NSY3ODAwOTk4MiU1QjQlNUQ9NTEzMTg4NDk3",

        cursor, results, mondata = get_json(kw, cursor)
        # save_data_mon('deviantart_cursor_json', [page, cursor])
        save_data_list('deviantart_images', mondata)

        # 遍历
        while cursor and page <= 500:
            # time.sleep(5)
            page += 1
            cursor, results, mondata = get_json(kw, cursor)
            print('print(page):', page,len(mondata))
            # save_data_mon('deviantart_cursor_json', [page, cursor])
            save_data_list('deviantart_images', mondata)
            if len(mondata) == 0:
                break
